# -*- coding: UTF-8 -*-
# @Project ：big-data 
# @File    ：招聘数据.py
# @Author  ：于金龙
# @IDE     ：PyCharm 
# @Date    ：2024/4/23 16:30
import pandas as pd
from gensim.models import LdaModel, CoherenceModel
from gensim import corpora, models, similarities
import gensim
import re
import jieba

data = pd.read_excel('./recruitment.xlsx')
# 重复值的数量
duplicated_num = data.duplicated().sum()
print(duplicated_num)

#  删除重复值
data.drop_duplicates(inplace=True)
# 删除职位描述为空的数据
data.dropna(subset=['职位描述'], inplace=True, axis=0, )
print(f'表格行列大小{data.shape}')


# 加载停用词
def stopwords_list(filepath):
    stopwords = []
    with open(filepath, 'r', encoding='utf-8') as file:
        for line in file:
            stopwords.append(line.strip())
    return stopwords


stopwords = stopwords_list('./stopwords.txt')
# 添加停用词
stopwords.extend(
    ['岗位职责', '负责', '优先', "\xa0", '负责', '公司', '能力', '岗位要求', "\r\n", "要求", "要求", "要求描述",
     '职责描述', '工作', '相关', '具备', '岗位', '\n', '\t', ',', 'and'])


def data_cleaning(text):
    # 去除特殊符号
    text = re.sub('[^\w\s]', '', text)
    con_list = jieba.cut(text)
    seg_list = [word for word in con_list if word not in stopwords]
    return seg_list


if __name__ == '__main__':

    data['职位描述分词'] = data['职位描述'].apply(data_cleaning)
    print(data['职位描述分词'][:5])

    # 文本向量化
    dictionary = corpora.Dictionary(data['职位描述分词'])
    corpus = [dictionary.doc2bow(i) for i in data['职位描述分词']]
    # 构架主题模型
    lda_model = LdaModel(corpus, num_topics=5, id2word=dictionary, passes=20)
    # 打印每个主题的n个关键词
    topics = lda_model.print_topics(num_topics=5, num_words=6)
    for topic in topics:
        print(topic)

    ida = []
    for i in lda_model.get_document_topics(corpus):
        listj = []
        for j in i:
            listj.append(j[1])
        bz = listj.index(max(listj))
        ida.append(i[bz][0])
    data['职位主题'] = ida
    print(data['职位主题'][:5])

    perplexity = lda_model.log_perplexity(corpus)
    print(f'困惑度：{perplexity}')

    coherence_model_lda = CoherenceModel(model=lda_model, texts=data['职位描述分词']
                                         , dictionary=dictionary, coherence='c_v')
    coherence_lda = coherence_model_lda.get_coherence()
    print(f'一致性：{coherence_lda}')
